name: Run Tutorials Nightly

on:
  workflow_dispatch: # Activate this workflow manually
  schedule:
    - cron: "0 0 * * *"

jobs:
  generate-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.generator.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - id: generator
        env:
          GH_TOKEN: ${{ github.token }}  
        run: |
          VERSION=$(gh api /repos/deepset-ai/haystack/releases/latest | jq -r .tag_name)
          NOTEBOOKS=$(python ./scripts/generate_matrix.py --haystack-version "$VERSION" --include-main)
          echo "matrix={\"include\":$NOTEBOOKS}" >> "$GITHUB_OUTPUT"

  run-tutorials:
    needs: generate-matrix
    runs-on: ubuntu-latest
    container: deepset/haystack:base-cpu-${{ matrix.haystack_version }}

    services:
      elasticsearch:
        image: elasticsearch:7.9.2
        env:
          discovery.type: "single-node"
          ES_JAVA_OPTS: "-Xms128m -Xmx256m"

    strategy:
      max-parallel: 2
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}

    env:
      HAYSTACK_TELEMETRY_ENABLED: "False"
      ELASTICSEARCH_HOST: "elasticsearch"
      HF_API_KEY: ${{ secrets.HF_API_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install dependencies
        # remove pip install pyzmq when this is resolved https://github.com/zeromq/pyzmq/issues/1764
        run: |
          apt-get update && apt-get install -y build-essential gcc libsndfile1 ffmpeg && rm -rf /var/lib/apt/lists/*
          pip install pyzmq==23.2.1
          pip install nbconvert ipython
          pip install "pyworld<=0.2.12" espnet espnet-model-zoo pydub
          pip install farm-haystack[pdf]

      - name: Install Haystack Extras text2speech dependencies
        run: |
          pip install farm-haystack-text2speech

      - name: Install Hugging Face datasets
        run: |
          pip install "datasets>=2.6.1"
      
      - name: Install ipywidgets for tutorial 24
        run: |
          pip install ipywidgets 

      - name: Convert notebook to Python
        run: |
          jupyter nbconvert --to python --RegexRemovePreprocessor.patterns '%%bash' ./tutorials/${{ matrix.notebook }}.ipynb

      - name: Run the converted notebook
        run: |
          python ./tutorials/${{ matrix.notebook }}.py

      - name: Send Failure to Datadog
        if: failure()
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "Tutorial ${{ matrix.notebook }} failed"
              text: "Branch ${{ github.ref_name }} tests failed"
              alert_type: "error"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "name:${{ matrix.notebook }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
